# -*- coding: utf-8 -*-
import node as node
import scrapy
from rest_framework import response
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from Douban.items import DoubanItem


class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['douban.com']
    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        # 定位节点列表
        node_list = response.xpath('//*[@id="content"]/div/div[1]/ol/Li/div/div[2]')# print(len(node_list))
        for node in node_list:
            item = DoubanItem()
            item['name'] = node.xpath('./div[1]/a/span[1]/text()').extract_first()
            item['score'] = node.xpath('./div[2]/div/span[2]/text()').extract_first()# strip用来去除字符串中的不可见字符
            item['info'] = node.xpath('./div[2]/p[1]/text()').extract_first().strip()
            item['desc'] = node.xpath('./div[2]/p[2]/span/text()').extract_first()

            yield item

            # 下一页
            next_url = 'https://movie.douban.com/top250' + response.xpath(
                '//*[@id="content"]/div/div[1]/div[2]/span[3]/a/@href').extract_first()
            yield scrapy.Request(
                url=next_url,
                callback=self.parse
            )
